var cheerio = require('cheerio')
var superagent = require('superagent')
var url = require('url')
var async = require('async')
var eventproxy = require('eventproxy')
var ep = new eventproxy()


require('superagent-proxy')(superagent);
var proxy = process.env.http_proxy || 'http://H2D28VDK85DDF49D:02C3E91B91DEC0A6@proxy.abuyun.com:9020';




var city = require('./city');
var citys=[]
for(var i=0;i<city.length;i++){
    citys.push(city[i].city)
}


var cnodeUrl = 'http://b2b.11467.com/'

superagent.get(cnodeUrl)
    .proxy(proxy)
    .end(function(err, res) {
        if (err) {
          return console.error(err)
        }
        var topicUrls = []
        var sanjiUrls=[]
        var urls=[]
        var city2=[]
        var $ = cheerio.load(res.text)

        for(var i=0;i<1;i++){
            city2.push($("#il .box.t5").eq(4).find(".boxcontent .listtxt a").eq(i).text()+'市')
            for(var j=0;j<citys.length;j++){
                if(citys[j]==city2[i]){
                    // 得到84个城市的url  actCity
                    topicUrls.push($("#il .box.t5").eq(4).find(".boxcontent .listtxt a").eq(i).attr("href")+'dir/old.htm')
                }
            }
        }


        ep.after('accessRepo', topicUrls.length, function(repos) {
            var oneRepo = repos.map(function (repo) {
                var repoUrl = repo[0]
                var repoHTMl = repo[1]
                var $ = cheerio.load(repoHTMl)
            })
        })

        topicUrls.forEach(function (repoUrl) {
            superagent.get(repoUrl)
            .proxy(proxy)
            .end(function (err, res) {
                var $$=cheerio.load(res.text)
                $$('.box.sidesubcat.t5 a').each(function (index, element) {
                    sanjiUrls.push($$(element).attr('href'))
                })
                sanji()
                console.log('【fetch ' + repoUrl + ' successful】');
                ep.emit('accessRepo', [repoUrl, res.text]);
            });
        })


        
        function sanji(){
            ep.after('accessRepo1', sanjiUrls.length, function(repos) {
                var oneRepo = repos.map(function (repo) {
                    var repoUrl = repo[0]
                    var repoHTMl = repo[1]
                    var $ = cheerio.load(repoHTMl)
                    var httpGitUrl = $('#logo h1').text()
                    console.log(httpGitUrl)
                    console.log(urls.length)
                    return ({
                        httpGitUrl: httpGitUrl
                    })
                })
            })

            sanjiUrls.forEach(function (repoUrl) {
                superagent.get(repoUrl)
                .proxy(proxy)
                .end(function (err, res) {
                    var $$=cheerio.load(res.text)
                    // $$('.box.sidesubcat.t5 a').each(function (index, element) {
                    //     urls.push($$(element).attr('href'))
                    // })
                    console.log('【fetch ' + repoUrl + ' successful】');
                    ep.emit('accessRepo1', [repoUrl, res.text]);
                });
            })
        }

        





        var concurrencyCount = 0
        var itemArr = []
        var fetchUrl = function(topicUrl, callback){
            var delay = parseInt((Math.random() * 10000000) % 2000, 10)
            concurrencyCount ++
            console.log('并发数：'+ concurrencyCount,'正在抓取：',topicUrl,'耗时：（ms）'+delay)
            superagent.get(topicUrl)
                // .proxy(proxy)
                .end(function(err,res){
                    var $ = cheerio.load(res.text)
                    for(var i=0;i<2;i++){
                        itemArr.push($(".box.sidesubcat.t5 a").eq(i).attr("href"))
                    }
                    
                })
            setTimeout(function(){
                concurrencyCount --
                callback(null,topicUrl)
            },1000+delay)
        }

        async.mapLimit(urls,5,
            function(topicUrl, callback){
                fetchUrl(topicUrl, callback)
            },
            function(err, result){
                console.log('final:')
                console.log(result)
                console.log(itemArr)
            })
    })